{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AirlineCode</th>\n",
       "      <th>FlightNumber</th>\n",
       "      <th>TailNumber</th>\n",
       "      <th>FT</th>\n",
       "      <th>SchedDepApt</th>\n",
       "      <th>SchedArrApt</th>\n",
       "      <th>SchedDepUtc</th>\n",
       "      <th>SchedArrUtc</th>\n",
       "      <th>ATD_UTC</th>\n",
       "      <th>ATA_UTC</th>\n",
       "      <th>...</th>\n",
       "      <th>arrival_delay</th>\n",
       "      <th>netarrival_delay</th>\n",
       "      <th>scheduled_flight_time</th>\n",
       "      <th>departure_week</th>\n",
       "      <th>departure_year</th>\n",
       "      <th>departure_dayofweek</th>\n",
       "      <th>time_inhours</th>\n",
       "      <th>SchedArrUtc_prevflt</th>\n",
       "      <th>TailNumber_prevflt</th>\n",
       "      <th>separationwith_prevflt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FD</td>\n",
       "      <td>1103</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HKT</td>\n",
       "      <td>UTP</td>\n",
       "      <td>2020-03-08 01:45:00+00:00</td>\n",
       "      <td>2020-03-08 03:00:00+00:00</td>\n",
       "      <td>2020-03-08 01:35:00+00:00</td>\n",
       "      <td>2020-03-08 02:52:00+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>-8</td>\n",
       "      <td>2</td>\n",
       "      <td>75</td>\n",
       "      <td>10</td>\n",
       "      <td>2020</td>\n",
       "      <td>7</td>\n",
       "      <td>1.750000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AK</td>\n",
       "      <td>6145</td>\n",
       "      <td>9M-AFD</td>\n",
       "      <td>Airbus A320</td>\n",
       "      <td>PEN</td>\n",
       "      <td>KUL</td>\n",
       "      <td>2020-02-18 00:40:00+00:00</td>\n",
       "      <td>2020-02-18 01:40:00+00:00</td>\n",
       "      <td>2020-02-18 00:50:00+00:00</td>\n",
       "      <td>2020-02-18 01:55:00+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>8</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>2020-03-08 03:00:00+00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AK</td>\n",
       "      <td>5188</td>\n",
       "      <td>9M-AFD</td>\n",
       "      <td>Airbus A320</td>\n",
       "      <td>KUL</td>\n",
       "      <td>SDK</td>\n",
       "      <td>2020-02-18 02:05:00+00:00</td>\n",
       "      <td>2020-02-18 05:05:00+00:00</td>\n",
       "      <td>2020-02-18 02:26:00+00:00</td>\n",
       "      <td>2020-02-18 05:20:00+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>15</td>\n",
       "      <td>-6</td>\n",
       "      <td>180</td>\n",
       "      <td>8</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>2.083333</td>\n",
       "      <td>2020-02-18 01:40:00+00:00</td>\n",
       "      <td>9M-AFD</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AK</td>\n",
       "      <td>5189</td>\n",
       "      <td>9M-AFD</td>\n",
       "      <td>Airbus A320</td>\n",
       "      <td>SDK</td>\n",
       "      <td>KUL</td>\n",
       "      <td>2020-02-18 05:30:00+00:00</td>\n",
       "      <td>2020-02-18 08:15:00+00:00</td>\n",
       "      <td>2020-02-18 05:45:00+00:00</td>\n",
       "      <td>2020-02-18 08:24:00+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>9</td>\n",
       "      <td>-6</td>\n",
       "      <td>165</td>\n",
       "      <td>8</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>2020-02-18 05:05:00+00:00</td>\n",
       "      <td>9M-AFD</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AK</td>\n",
       "      <td>6128</td>\n",
       "      <td>9M-AFD</td>\n",
       "      <td>Airbus A320</td>\n",
       "      <td>KUL</td>\n",
       "      <td>PEN</td>\n",
       "      <td>2020-02-18 09:40:00+00:00</td>\n",
       "      <td>2020-02-18 10:35:00+00:00</td>\n",
       "      <td>2020-02-18 09:30:00+00:00</td>\n",
       "      <td>2020-02-18 10:28:00+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>-7</td>\n",
       "      <td>3</td>\n",
       "      <td>55</td>\n",
       "      <td>8</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>9.666667</td>\n",
       "      <td>2020-02-18 08:15:00+00:00</td>\n",
       "      <td>9M-AFD</td>\n",
       "      <td>85.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  AirlineCode  FlightNumber TailNumber           FT SchedDepApt SchedArrApt  \\\n",
       "0          FD          1103        NaN          NaN         HKT         UTP   \n",
       "1          AK          6145     9M-AFD  Airbus A320         PEN         KUL   \n",
       "2          AK          5188     9M-AFD  Airbus A320         KUL         SDK   \n",
       "3          AK          5189     9M-AFD  Airbus A320         SDK         KUL   \n",
       "4          AK          6128     9M-AFD  Airbus A320         KUL         PEN   \n",
       "\n",
       "                 SchedDepUtc                SchedArrUtc  \\\n",
       "0  2020-03-08 01:45:00+00:00  2020-03-08 03:00:00+00:00   \n",
       "1  2020-02-18 00:40:00+00:00  2020-02-18 01:40:00+00:00   \n",
       "2  2020-02-18 02:05:00+00:00  2020-02-18 05:05:00+00:00   \n",
       "3  2020-02-18 05:30:00+00:00  2020-02-18 08:15:00+00:00   \n",
       "4  2020-02-18 09:40:00+00:00  2020-02-18 10:35:00+00:00   \n",
       "\n",
       "                     ATD_UTC                    ATA_UTC  ... arrival_delay  \\\n",
       "0  2020-03-08 01:35:00+00:00  2020-03-08 02:52:00+00:00  ...            -8   \n",
       "1  2020-02-18 00:50:00+00:00  2020-02-18 01:55:00+00:00  ...            15   \n",
       "2  2020-02-18 02:26:00+00:00  2020-02-18 05:20:00+00:00  ...            15   \n",
       "3  2020-02-18 05:45:00+00:00  2020-02-18 08:24:00+00:00  ...             9   \n",
       "4  2020-02-18 09:30:00+00:00  2020-02-18 10:28:00+00:00  ...            -7   \n",
       "\n",
       "  netarrival_delay scheduled_flight_time  departure_week  departure_year  \\\n",
       "0                2                    75              10            2020   \n",
       "1                5                    60               8            2020   \n",
       "2               -6                   180               8            2020   \n",
       "3               -6                   165               8            2020   \n",
       "4                3                    55               8            2020   \n",
       "\n",
       "  departure_dayofweek time_inhours        SchedArrUtc_prevflt  \\\n",
       "0                   7     1.750000                        NaN   \n",
       "1                   2     0.666667  2020-03-08 03:00:00+00:00   \n",
       "2                   2     2.083333  2020-02-18 01:40:00+00:00   \n",
       "3                   2     5.500000  2020-02-18 05:05:00+00:00   \n",
       "4                   2     9.666667  2020-02-18 08:15:00+00:00   \n",
       "\n",
       "  TailNumber_prevflt separationwith_prevflt  \n",
       "0                NaN                    NaN  \n",
       "1                NaN                    NaN  \n",
       "2             9M-AFD                   25.0  \n",
       "3             9M-AFD                   25.0  \n",
       "4             9M-AFD                   85.0  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df = pd.read_csv('preprocessed_data.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "171\n",
      "172\n"
     ]
    }
   ],
   "source": [
    "uniqueDepAirports = df['SchedDepApt'].unique()\n",
    "uniqueArrAirports = df['SchedArrApt'].unique()\n",
    "\n",
    "uniqueAirlineCode = df['AirlineCode'].unique()\n",
    "\n",
    "uniqueTailNumber = df['TailNumber'].unique()\n",
    "\n",
    "uniqueFT = df['FT'].unique()\n",
    "\n",
    "uniquePrevTail = df['TailNumber_prevflt'].unique()\n",
    "\n",
    "\n",
    "print(len(uniqueDepAirports))\n",
    "print(len(uniqueArrAirports))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "172\n"
     ]
    }
   ],
   "source": [
    "AirportLists = set(uniqueDepAirports).union(set(uniqueArrAirports))\n",
    "AirportLists = list(set(AirportLists))\n",
    "\n",
    "print(len(AirportLists))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "aptDict = {}\n",
    "revApt = {}\n",
    "index = 0\n",
    "\n",
    "for apt in AirportLists:\n",
    "    aptDict[apt] = index\n",
    "    revApt[index] = apt\n",
    "    index += 1\n",
    "\n",
    "index = 0   \n",
    "codeDict = {}\n",
    "revCode = {}\n",
    "for airCode in uniqueAirlineCode:\n",
    "    codeDict[airCode] = index\n",
    "    revCode[index] = airCode\n",
    "    index += 1\n",
    "    \n",
    "index = 0\n",
    "tailDict = {}\n",
    "revTail = {}\n",
    "\n",
    "for tail in uniqueTailNumber:\n",
    "    tailDict[tail] = index\n",
    "    revTail[index] = tail\n",
    "    index += 1\n",
    "    \n",
    "index = 0\n",
    "ftDict = {}\n",
    "revFT = {}\n",
    "\n",
    "for ft in uniqueFT:\n",
    "    ftDict[ft] = index\n",
    "    revFT[index] = ft\n",
    "    index += 1\n",
    "    \n",
    "index = 0\n",
    "prevDict = {}\n",
    "revPrev = {}   \n",
    "\n",
    "for prev in uniquePrevTail:\n",
    "    prevDict[prev] = index\n",
    "    revPrev[index] = prev\n",
    "    index += 1\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AirlineCode       0\n",
       "FlightNumber      0\n",
       "TailNumber        0\n",
       "FT                0\n",
       "SchedDepApt       0\n",
       "SchedArrApt       0\n",
       "SchedDep_Year     0\n",
       "SchedDep_Month    0\n",
       "SchedDep_Day      0\n",
       "SchedDep_Time     0\n",
       "SchedArr_Year     0\n",
       "SchedArr_Month    0\n",
       "SchedArr_Day      0\n",
       "SchedArr_Time     0\n",
       "Total_PAX         0\n",
       "Baggage_pieces    0\n",
       "DistanceGC        0\n",
       "arrival_delay     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['SchedDep_Year'] = pd.DatetimeIndex(df['SchedDepUtc']).year\n",
    "df['SchedDep_Month'] = pd.DatetimeIndex(df['SchedDepUtc']).month\n",
    "df['SchedDep_Day'] = pd.DatetimeIndex(df['SchedDepUtc']).day\n",
    "df['SchedDep_Time'] = np.round(pd.DatetimeIndex(df['SchedDepUtc']).hour + (pd.DatetimeIndex(df['SchedDepUtc']).minute / 60))\n",
    "\n",
    "\n",
    "df['SchedArr_Year'] = pd.DatetimeIndex(df['SchedArrUtc']).year\n",
    "df['SchedArr_Month'] = pd.DatetimeIndex(df['SchedArrUtc']).month\n",
    "df['SchedArr_Day'] = pd.DatetimeIndex(df['SchedArrUtc']).day\n",
    "df['SchedArr_Time'] = np.round(pd.DatetimeIndex(df['SchedArrUtc']).hour + (pd.DatetimeIndex(df['SchedArrUtc']).minute / 60))\n",
    "\n",
    "\n",
    "# df['FlightDoorClose_Year'] = pd.DatetimeIndex(df['FlightDoorClose_UTC']).year\n",
    "# df['FlightDoorClose_Month'] = pd.DatetimeIndex(df['FlightDoorClose_UTC']).month\n",
    "# df['FlightDoorClose_Day'] = pd.DatetimeIndex(df['FlightDoorClose_UTC']).day\n",
    "# df['FlightDoorClose_Time'] = np.round(pd.DatetimeIndex(df['FlightDoorClose_UTC']).hour + (pd.DatetimeIndex(df['FlightDoorClose_UTC']).minute / 60))\n",
    "\n",
    "\n",
    "# df['FlightTakeOff_Year'] = pd.DatetimeIndex(df['FlightTakeOff_UTC']).year\n",
    "# df['FlightTakeOff_Month'] = pd.DatetimeIndex(df['FlightTakeOff_UTC']).month\n",
    "# df['FlightTakeOff_Day'] = pd.DatetimeIndex(df['FlightTakeOff_UTC']).day\n",
    "# df['FlightTakeOff_Time'] = np.round(pd.DatetimeIndex(df['FlightTakeOff_UTC']).hour + (pd.DatetimeIndex(df['FlightTakeOff_UTC']).minute / 60))\n",
    "\n",
    "\n",
    "df = df[[\"AirlineCode\", \"FlightNumber\", \"TailNumber\", \"FT\", \"SchedDepApt\", \"SchedArrApt\", \"SchedDep_Year\", \"SchedDep_Month\", \"SchedDep_Day\", \"SchedDep_Time\",  \"SchedArr_Year\", \"SchedArr_Month\", \"SchedArr_Day\", \"SchedArr_Time\", \"Total_PAX\", \"Baggage_pieces\", \"DistanceGC\", \"arrival_delay\"]]\n",
    "\n",
    "\n",
    "df.head()\n",
    "\n",
    "\n",
    "df = df.fillna({\"TailNumber\":'9M-AFD', \"FT\": \"Airbus A320\",\"Total_PAX\": 140, \"Baggage_pieces\": 63})\n",
    "df.isnull().sum()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FlightNumber</th>\n",
       "      <th>SchedDep_Year</th>\n",
       "      <th>SchedDep_Month</th>\n",
       "      <th>SchedDep_Day</th>\n",
       "      <th>SchedDep_Time</th>\n",
       "      <th>SchedArr_Year</th>\n",
       "      <th>SchedArr_Month</th>\n",
       "      <th>SchedArr_Day</th>\n",
       "      <th>SchedArr_Time</th>\n",
       "      <th>Total_PAX</th>\n",
       "      <th>...</th>\n",
       "      <th>SchedArrApt_URT</th>\n",
       "      <th>SchedArrApt_UTH</th>\n",
       "      <th>SchedArrApt_UTP</th>\n",
       "      <th>SchedArrApt_VCA</th>\n",
       "      <th>SchedArrApt_VNS</th>\n",
       "      <th>SchedArrApt_VTE</th>\n",
       "      <th>SchedArrApt_VTZ</th>\n",
       "      <th>SchedArrApt_WUH</th>\n",
       "      <th>SchedArrApt_XIY</th>\n",
       "      <th>SchedArrApt_YIA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1103</td>\n",
       "      <td>2020</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>3.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6145</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>2.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5188</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>5.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5189</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>8.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6128</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>10.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>11.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 649 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   FlightNumber  SchedDep_Year  SchedDep_Month  SchedDep_Day  SchedDep_Time  \\\n",
       "0          1103           2020               3             8            2.0   \n",
       "1          6145           2020               2            18            1.0   \n",
       "2          5188           2020               2            18            2.0   \n",
       "3          5189           2020               2            18            6.0   \n",
       "4          6128           2020               2            18           10.0   \n",
       "\n",
       "   SchedArr_Year  SchedArr_Month  SchedArr_Day  SchedArr_Time  Total_PAX  ...  \\\n",
       "0           2020               3             8            3.0      134.0  ...   \n",
       "1           2020               2            18            2.0       90.0  ...   \n",
       "2           2020               2            18            5.0       98.0  ...   \n",
       "3           2020               2            18            8.0      130.0  ...   \n",
       "4           2020               2            18           11.0      115.0  ...   \n",
       "\n",
       "   SchedArrApt_URT  SchedArrApt_UTH  SchedArrApt_UTP  SchedArrApt_VCA  \\\n",
       "0                0                0                1                0   \n",
       "1                0                0                0                0   \n",
       "2                0                0                0                0   \n",
       "3                0                0                0                0   \n",
       "4                0                0                0                0   \n",
       "\n",
       "   SchedArrApt_VNS  SchedArrApt_VTE  SchedArrApt_VTZ  SchedArrApt_WUH  \\\n",
       "0                0                0                0                0   \n",
       "1                0                0                0                0   \n",
       "2                0                0                0                0   \n",
       "3                0                0                0                0   \n",
       "4                0                0                0                0   \n",
       "\n",
       "   SchedArrApt_XIY  SchedArrApt_YIA  \n",
       "0                0                0  \n",
       "1                0                0  \n",
       "2                0                0  \n",
       "3                0                0  \n",
       "4                0                0  \n",
       "\n",
       "[5 rows x 649 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.get_dummies(df, columns=[\"AirlineCode\", \"TailNumber\", \"FT\", \"SchedDepApt\", \"SchedArrApt\"])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FlightNumber</th>\n",
       "      <th>SchedDep_Year</th>\n",
       "      <th>SchedDep_Month</th>\n",
       "      <th>SchedDep_Day</th>\n",
       "      <th>SchedDep_Time</th>\n",
       "      <th>SchedArr_Year</th>\n",
       "      <th>SchedArr_Month</th>\n",
       "      <th>SchedArr_Day</th>\n",
       "      <th>SchedArr_Time</th>\n",
       "      <th>Total_PAX</th>\n",
       "      <th>...</th>\n",
       "      <th>SchedArrApt_URT</th>\n",
       "      <th>SchedArrApt_UTH</th>\n",
       "      <th>SchedArrApt_UTP</th>\n",
       "      <th>SchedArrApt_VCA</th>\n",
       "      <th>SchedArrApt_VNS</th>\n",
       "      <th>SchedArrApt_VTE</th>\n",
       "      <th>SchedArrApt_VTZ</th>\n",
       "      <th>SchedArrApt_WUH</th>\n",
       "      <th>SchedArrApt_XIY</th>\n",
       "      <th>SchedArrApt_YIA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1103</td>\n",
       "      <td>2020</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>3.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6145</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>2.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5188</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>5.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5189</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>8.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6128</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>10.0</td>\n",
       "      <td>2020</td>\n",
       "      <td>2</td>\n",
       "      <td>18</td>\n",
       "      <td>11.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 649 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   FlightNumber  SchedDep_Year  SchedDep_Month  SchedDep_Day  SchedDep_Time  \\\n",
       "0          1103           2020               3             8            2.0   \n",
       "1          6145           2020               2            18            1.0   \n",
       "2          5188           2020               2            18            2.0   \n",
       "3          5189           2020               2            18            6.0   \n",
       "4          6128           2020               2            18           10.0   \n",
       "\n",
       "   SchedArr_Year  SchedArr_Month  SchedArr_Day  SchedArr_Time  Total_PAX  ...  \\\n",
       "0           2020               3             8            3.0      134.0  ...   \n",
       "1           2020               2            18            2.0       90.0  ...   \n",
       "2           2020               2            18            5.0       98.0  ...   \n",
       "3           2020               2            18            8.0      130.0  ...   \n",
       "4           2020               2            18           11.0      115.0  ...   \n",
       "\n",
       "   SchedArrApt_URT  SchedArrApt_UTH  SchedArrApt_UTP  SchedArrApt_VCA  \\\n",
       "0                0                0                1                0   \n",
       "1                0                0                0                0   \n",
       "2                0                0                0                0   \n",
       "3                0                0                0                0   \n",
       "4                0                0                0                0   \n",
       "\n",
       "   SchedArrApt_VNS  SchedArrApt_VTE  SchedArrApt_VTZ  SchedArrApt_WUH  \\\n",
       "0                0                0                0                0   \n",
       "1                0                0                0                0   \n",
       "2                0                0                0                0   \n",
       "3                0                0                0                0   \n",
       "4                0                0                0                0   \n",
       "\n",
       "   SchedArrApt_XIY  SchedArrApt_YIA  \n",
       "0                0                0  \n",
       "1                0                0  \n",
       "2                0                0  \n",
       "3                0                0  \n",
       "4                0                0  \n",
       "\n",
       "[5 rows x 649 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df_model = pd.get_dummies(df_model, columns=[\"AirlineCode\", \"TailNumber\", \"FT\", \"SchedDepApt\", \"SchedArrApt\", \"TailNumber_prevflt\"])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9480,)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##Create the partition of the current data into 80% training data and 20% testing data\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "train_x, test_x, train_y, test_y = train_test_split(df.drop('arrival_delay', axis=1), df['arrival_delay'], test_size=0.2, random_state=42)\n",
    "\n",
    "train_x.shape\n",
    "test_x.shape\n",
    "train_y.shape\n",
    "test_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a RandomForestRegressor object to be trained on the training data\n",
    "\n",
    "#According to the documentation of the Scikit-learn, the most important parameters in the Random Forest are number of trees\n",
    "#i.e n_estimators and number of features selected for splitting at each node\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "regressor = RandomForestRegressor(n_estimators=100, random_state=0)\n",
    "#regressor.fit(train_x, train_y) #Training the model over the training data with the below mentioned parameters\n",
    "\n",
    "model = regressor.fit(train_x, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.18142322562073232\n"
     ]
    }
   ],
   "source": [
    "score = model.score(test_x, test_y)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ -1.19,  -6.13, -11.  , ...,  -1.58, -10.97, -14.73])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regressor_pred_a = model.predict(test_x)\n",
    "regressor_pred_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Errors count less than 10 is 6058\n",
      "percent time  63.90295358649789\n"
     ]
    }
   ],
   "source": [
    "pos = 0\n",
    "sample = []\n",
    "count = 0\n",
    "\n",
    "for index, value in test_y.items():\n",
    "    sample.append(abs(value - regressor_pred_a[pos])) \n",
    "    count += 1\n",
    "    pos += 1\n",
    "    \n",
    "lessThanTen = 0\n",
    "\n",
    "for i in range(0, len(sample)):\n",
    "    if sample[i] <= 10:\n",
    "        lessThanTen += 1\n",
    "        \n",
    "print(\"Errors count less than 10 is\", lessThanTen)\n",
    "\n",
    "percentError = (lessThanTen * 100) / len(test_y)\n",
    "print(\"percent time \",percentError)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KFold(n_splits=10, random_state=None, shuffle=False)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "X = df.drop('arrival_delay', axis=1)\n",
    "Y = df['arrival_delay']\n",
    "\n",
    "kf = KFold(n_splits=10)\n",
    "kf.get_n_splits(X)\n",
    "\n",
    "print(kf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "0.17204161100844761\n",
      "2\n",
      "0.850903390364304\n",
      "3\n",
      "0.8970187957503127\n",
      "4\n",
      "0.8855139783944763\n",
      "5\n",
      "0.9101770072073081\n",
      "6\n",
      "0.8742938923816954\n",
      "7\n",
      "0.8791692987867435\n",
      "8\n",
      "0.8726595973792735\n",
      "9\n",
      "0.898783844816123\n",
      "10\n",
      "0.23320792118619615\n",
      "Final mean score is nan\n",
      "AVG of percent time error less than 10 is  85.73665374756375\n",
      "AVG of percent time error less than 15 is  91.97785900815836\n",
      "AVG of percent time error less than 20 is  94.9929982201732\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/numpy/core/fromnumeric.py:3335: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/opt/conda/lib/python3.7/site-packages/numpy/core/_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "scoreArrayTrain = []\n",
    "scoreArrayTest = []\n",
    "\n",
    "error_10 = []\n",
    "error_15 = []\n",
    "error_20 = []\n",
    "count = 1\n",
    "\n",
    "model = []\n",
    "\n",
    "\n",
    "for train_index, test_index in kf.split(X):\n",
    "    #print(type(train_index))\n",
    "    print(count)\n",
    "    train_X = X[train_index[0]:train_index[-1] + 1]\n",
    "    train_Y = Y[train_index[0]:train_index[-1] + 1]\n",
    "    test_X = X[test_index[0]:test_index[-1] + 1]\n",
    "    test_Y = Y[test_index[0]:test_index[-1] + 1]\n",
    "    \n",
    "    regressor.fit(train_X, train_Y)\n",
    "    \n",
    "#     scoreTrain = xg_reg.score(train_x, train_y)\n",
    "#     print(scoreTrain)\n",
    "#     scoreArrayTrain.append(scoreTrain)\n",
    "    \n",
    "    scoreTest = regressor.score(test_X, test_Y)\n",
    "    print(scoreTest)\n",
    "    \n",
    "    model.append(regressor)\n",
    "    \n",
    "    regressor_pred_a = regressor.predict(test_X)\n",
    "    \n",
    "    pos = 0\n",
    "#     sample = []\n",
    "#     count = 0\n",
    "    \n",
    "    lessThanTen = 0\n",
    "    lessThanFifteen = 0\n",
    "    lessThanTwenty = 0\n",
    "    \n",
    "    for index, value in test_Y.items():\n",
    "        #sample.append(abs(value - regressor_pred_a[pos])) \n",
    "        #count += 1\n",
    "        \n",
    "        if abs(value - regressor_pred_a[pos]) <= 10:\n",
    "            lessThanTen += 1\n",
    "        if abs(value - regressor_pred_a[pos]) <= 15:\n",
    "            lessThanFifteen += 1\n",
    "        if abs(value - regressor_pred_a[pos]) <= 20:\n",
    "            lessThanTwenty += 1\n",
    "            \n",
    "        pos += 1\n",
    "    \n",
    "    \n",
    "\n",
    "#     for i in range(0, len(sample)):\n",
    "#         if sample[i] <= 10:\n",
    "#             lessThanTen += 1\n",
    "#         elif sample[i] <= 15:\n",
    "#             lessThanFifteen += 1\n",
    "#         elif sample[i] <= 20:\n",
    "#             lessThanTwenty += 1\n",
    "        \n",
    "    #print(\"Errors count less than 10 is\", lessThanTen)\n",
    "\n",
    "    percentError_10 = (lessThanTen * 100) / len(test_Y)\n",
    "    error_10.append(percentError_10)\n",
    "    \n",
    "    percentError_15 = (lessThanFifteen * 100) / len(test_Y)\n",
    "    error_15.append(percentError_15)\n",
    "    \n",
    "    percentError_20 = (lessThanTwenty * 100) / len(test_Y)\n",
    "    error_20.append(percentError_20)\n",
    "    \n",
    "    count += 1    \n",
    "    #print(\"percent time \",percentError)    \n",
    "    train_X = []\n",
    "    train_Y = []\n",
    "    test_X = []\n",
    "    test_Y = []\n",
    "    \n",
    "    \n",
    "print(\"Final mean score is\", np.mean(scoreArrayTest))  \n",
    "print(\"AVG of percent time error less than 10 is \", np.mean(error_10))\n",
    "print(\"AVG of percent time error less than 15 is \", np.mean(error_15))\n",
    "print(\"AVG of percent time error less than 20 is \", np.mean(error_20))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['newSavedRF_ArrDel.dat']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from joblib import dump\n",
    "\n",
    "dump(model[4], \"newSavedRF_ArrDel.dat\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8225306763141355\n"
     ]
    }
   ],
   "source": [
    "from joblib import load\n",
    "\n",
    "loadedModel = load(\"newSavedRF_ArrDel.dat\")\n",
    "\n",
    "print(loadedModel.score(test_x, test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "name": "common-cpu.m50",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m50"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
